package com.dtkavin.spark.partation

import org.apache.spark.{Partitioner, SparkConf, SparkContext}
import java.net.URL

/**
  * Created by IntelliJ IDEA.
  * Programmer : John Zn
  * Date : 2016/4/14 0014
  * Time : 18:23
  * Discribtion : 
  */
case class LogPartition() extends Partitioner {
  override def numPartitions: Int = 3

  override def getPartition(key: Any): Int = {
    key match {
      case str if str.toString.contains("java") => 1
      case _ => 0
    }
  }
}

object LogPartition {
  def main(args: Array[String]) {
    val conf = new SparkConf()
    val sc = new SparkContext("local", "LogPartition", conf)
//    sc.textFile("G:\\BigData\\lessions\\spark\\spark-day2\\itcast.log").map {
//      x => {
//        val arr = x.split("\t")
//        val host = new URL(arr(1)).getHost
//        (host, arr(1))
//      }
//    }.partitionBy(LogPartition()).foreachPartition { case (k, v) => {
//      println(Thread.currentThread().getId + ". " + k + ", " + v)
//    }
//    }
    sc.stop()
  }
}
